#coding:utf-8
'''
Created on 2015年4月22日

@author: LY
'''
from xml.dom import minidom
import re
import os
import sys

reload(sys)
sys.setdefaultencoding('utf8')

class LogParser(object):
    rootDir = "E:\\data"
    xmlDir = "D:\\git\\euler_etl\\app\\dataclean\\"  
    
    def parseLog(self, logPath, xmlConfigPath, date):
        xmlFile = self.xmlDir + os.path.sep + xmlConfigPath
        print xmlFile
        # xmlFile = xmlConfigPath
        if not os.path.exists(xmlFile):
            print "XML 配置文件   %s 不存在" %(xmlFile)
            return False
               
        indic = readXml(xmlFile,date, self.rootDir) 
        
        srcInfile = indic["infile"]
        srcType = indic["iftype"]
        srcSysname = indic["sysname"]
        print srcInfile,srcType,srcSysname
        
        if str(logPath) != str(srcInfile):
            print "警告 ：文件  %s 未配置  %s " %(xmlFile, logPath)
            return False
        
        srcInfileHandle = open(srcInfile,'rb')
        
        fileBypass = self.rootDir + os.path.sep + "fail" + os.path.sep + "bypass" + os.path.sep + str(date) + os.path.sep + srcSysname +  os.path.sep + srcSysname + "_" + os.path.basename(srcInfile).split('.')[0] + ".bypass"
        reportFile = self.rootDir + os.path.sep + "complete" + os.path.sep + date +  os.path.sep + srcSysname + os.path.sep + srcSysname + "_" + os.path.basename(srcInfile).split('.')[0] + ".report"
        #fileBypass = "%s\\fail\\bypass\\%s\\%s\\%s_%s.bypass" %(self.rootDir, date, srcSysname, srcSysname, os.path.basename(srcInfile).split('.')[0])
        #reportFile = "%s\\complete\\%s\\%s\\%s_%s.report" %(self.rootDir, date, srcSysname, srcSysname, os.path.basename(srcInfile).split('.')[0])
        
        makeFileDir(fileBypass)
        makeFileDir(reportFile)
        fbHandle = open(fileBypass,'w')
        reportHandle = open(reportFile,'w+')
        
        # 保存输出日志各文件句柄
        tmpFileHandle={}
        
        for fh in indic["fileHandle"]:
            for outFile,shortName in fh.items():
                makeFileDir(outFile)
                shortName = open(outFile,'w')
                tmpFileHandle[outFile] = shortName  
            
        # print "解析日志 ：'%s' 日志所属类型：'%s' \n" %(str(srcInfile), str(srcSysname))
     
        # 定义记录解析文件详细情况的字典
        countLines = {
            "totallines" : 0,
            "useless"    : 0
                      }
        # 清洗后匹配到日志关键词的行数
        countIn = {}
        # 匹配到长正则表达式的行数
        countOut = {}
    
        for line in srcInfileHandle:
            # 无用的数据行标志，为0表示无用的，为1表示解析到
            flag = 0
            countLines["totallines"] += 1
            line = line.strip()
            
            for logName in indic["fields"]:
                # logAlas = logName["fname"]
                shortName = logName["shortname"]
                logRegular = logName["regular"]
                ShortRegular = logName["short_reg"]
                outFile = logName["outfile"]
                errFile =  logName["errfile"]
                logDelim = logName["delim"]
            
                
                shortMatch = re.search(r'%s' %(ShortRegular), line)
        
                # 匹配到正则关键词
                if shortMatch:
        
                    flag = 1
                    
                    if countIn.has_key(shortName):
                        countIn[shortName] += 1
                    else:
                        countIn[shortName] = 1
                    
                    long_matchs = re.match(logRegular, line)
                    
                    # 匹配到长关键词
                    if long_matchs:
                        if countOut.has_key(shortName):
                            countOut[shortName] += 1
                        else:
                            countOut[shortName] = 1
                            
                        results = list(long_matchs.groups())
                        results[0] = changeTimeFormat(date, results[0])
                        
                        if re.match(r'\\t',logDelim):
                            logDelim = '\t'
    
                        msg = logDelim.join(results)
                        # print "write %s to file : %s" %(msg, outFile)
                        tmpFileHandle[outFile].write(msg + '\n')
                    # 匹配到正则关键词但是匹配不到长正则
                    else:
                        tmpFileHandle[errFile].write(line + '\n')
                        # print "write %s to file : %s" %(line, errFile)
                    break
                # 未匹配到时进行下一个日志正则的匹配
                else:
                    continue
            
            # 匹配到本行，直接进行下一行的解析    
            if flag:
                continue
            # 
            else:
                # print "write %s to bypass file %s" %(line, fileBypass)
                countLines["useless"] += 1
                fbHandle.write(line + '\n')
        
        for filePath in tmpFileHandle.keys():
            tmpFileHandle[filePath].close()
        
        details = []
        for inKey,inValue in countIn.items():
            if inKey not in countOut:
                countOut[inKey] = 0
            # print str(inKey) + ": [总行数：%d 正常匹配: %d 异常匹配: %d]" %(int(inValue), int(countOut[inKey]), int(inValue - countOut[inKey]))
            details.append(str(inKey) + "(总行数：%d 正常匹配: %d 异常匹配: %d)" %(int(inValue), int(countOut[inKey]), int(inValue - countOut[inKey])))   
        reportHandle.write("文件%s解析情况: 输入文件总行数 : %d 输入文件匹配行数： %d 其中[ %s ] 输入文件未匹配行数： %d" %(srcInfile, countLines["totallines"], countLines["totallines"] - countLines["useless"], '\t'.join(details), countLines["useless"]))
        # print '\t'.join(details)
        # print srcInfile, countLines["totallines"], countLines["totallines"] - countLines["useless"], '\t'.join(details), countLines["useless"]
        fbHandle.close()
        srcInfileHandle.close()
        reportHandle.close()
        
        return True
def replaceString(path, oStr, toStr):
    """
                    字符串替换函数
    """
    return path.replace(oStr,toStr)
         
def changeTimeFormat(date, hour):
    """
                  格式化时间函数  
    """
    if re.match(r'\d+:\d+:\d+', hour):
        return str(date) + ' ' + hour
    else:
        return hour
    
def makeFileDir(objFile):
    """"
                    给定一个文件，判断该文件所在目录是否存在，不存在创建
    """
    if not os.path.exists(os.path.dirname(objFile)):
        os.makedirs(os.path.dirname(objFile))

def readXml(xmlConf,date, rootDir):
    """
                        解析给定xml文件，返回一个包含属性元素字典的列表
    """
    xmldoc = minidom.parse(xmlConf)
    source = xmldoc.getElementsByTagName("source")[0]
    
    infile = replaceString(source.attributes["infile"].value, '#Y-m-d#', str(date))
    # 清洗文件的类型 0:普通日志文件  1:DB文件
    iftype = source.attributes["type"].value
    # 日志系统名
    sysname = source.attributes["sysname"].value
    # 清洗后的文件名列表
    fields = source.getElementsByTagName("field")
    # 存放清洗后日志文件名(包括所有属性一个字典)的列表
    fieldList = []
    fileHandle = []
    # 定义文件句柄字典

    for field in fields:
        fname = field.attributes["name"].value
        shortName = field.attributes["shortname"].value
        regular = field.attributes["regular"].value
        short_reg = field.attributes["short_reg"].value
        # outdir = replaceString(field.attributes["outdir"].value,'#Y-m-d#', str(date))
        delim = field.attributes["delim"].value
        
        outdir = rootDir + os.path.sep +  "complete" +  os.path.sep  + str(date) + os.path.sep + sysname
        errdir = rootDir + os.path.sep + "fail" + os.path.sep + "error" + os.path.sep + str(date) +  sysname 
        
        outfile = outdir + os.path.sep + fname
        errfile = errdir + os.path.sep + shortName + ".err"
        
        fh_file = {
             outfile : shortName,
             errfile :  shortName + '_' + 'err'            
        }

        fileHandle.append(fh_file)
        
        fdict = {
             "fname"     : fname,
             "shortname" : shortName,
             "regular"   : regular,
             "short_reg" : short_reg,
             "outfile"   : outfile,
             "errfile"   : errfile,
             "delim"     : delim
             }

        fieldList.append(fdict)        
    # 定义一个带有属性的infile文件字典
    sdict = {
             "infile"    : infile,
             "iftype"    : iftype, 
             "sysname"   : sysname,     
             "fields"    : fieldList,
             "fileHandle": fileHandle            
             }

    return sdict

if __name__ == "__main__":
    logParser = LogParser()
    logParser.parseLog("E:\\data\\receive\\2015-04-23\\CY_UC\\stdout.log.2015-04-23", "UC_STDOUT.xml", "2015-04-23")
    
        